# www.xinhuanet.com
# ddosakura: Template for mods

from mod.sub_mod.xinhuanet import func
from tqdm import tqdm

# 必须有
target_name = '新华网'
target_url = 'http://www.xinhuanet.com/'

def mod_entry(insert):
    l = func.init(target_url)
    # 测试用例
    # l = [
    #     'http://www.xinhuanet.com/world/2018-12/09/c_1123828263.htm',
    #     'http://www.xinhuanet.com/world/2018-12/10/c_1210011775.htm',
    #     'http://www.xinhuanet.com/world/2018-12/10/c_1210011775.htm',
    #     'http://m.xinhuanet.com/video/2018-12/10/c_1210011836.htm',
    # ]
    print('开始深度扫描')
    data = []
    for i in tqdm(range(0, len(l))):
        item = l[i]
        try:
            row = func.getContent(item)
            if row != None:
                data.append(row)
        except:
            pass
    print('深度扫描完成 获得资讯', len(data))

    print('查重过滤开始')
    data = func.cleanData(data)

    if len(data) > 0:
        # print(data)
        insert(data)
        print(len(data), '新资讯已入库')
    else:
        print('未增加新资讯')
    """
    insert([{

    }, {

    }])
    """
